package com.easy.stock.spider.browser;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Properties;
import java.util.Set;
import org.openqa.selenium.By;
import org.openqa.selenium.WebElement;
import org.springframework.beans.factory.annotation.Autowired;

import com.easy.stock.dao.ConfigDao;
import com.easy.stock.dao.StockFeedItemDao;
import com.easy.stock.dao.StockInfoDao;
import com.easy.stock.domain.ConfigItem;
import com.easy.stock.domain.StockFeedItem;
import com.easy.stock.domain.StockInfo;
import com.easy.stock.util.Constants;

public class StockTwitsSpider extends BaseSeleniumSpider {
	
	private static final String BULLISH_TAG = "Bullish";
	private static final String BEARISH_TAG = "Bearish"; 

	@Autowired
	private StockInfoDao stockInfoDao;
	
	@Autowired
	private StockFeedItemDao stockFeedItemDao;
	
	@Autowired
	private ConfigDao configDao;
	
	private String baseUrl;
	private int maxRetryNum=3;
	private Date defaultLastCrawlTs=null;
	
	public void setup(Properties props) throws Exception {
		SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
		defaultLastCrawlTs = sdf.parse(props.getProperty("day"));
	}
	

	public int crawl() throws Exception {				
		// step1, 按个股票抓取信息
		restartDriver();
		int limit = 100;
		int offset = 0;
		List<StockInfo> stocks = null;
		while(!(stocks=stockInfoDao.getStockInfoList(offset, limit)).isEmpty()) {
			
			for(StockInfo stock : stocks) {
				Date lastCrawlTs = defaultLastCrawlTs;
				
				// step 2.1 先判断是否已经抓取过了
				ConfigItem cfgItem = configDao.getConfig(Constants.STOCKTWIS, "ts", stock.getId());
				if(cfgItem!=null) {
					if(cfgItem.getDate().after(defaultLastCrawlTs))
						lastCrawlTs = cfgItem.getDate();
				} else {
					cfgItem = new ConfigItem();
					cfgItem.setApp(Constants.STOCKTWIS);
					cfgItem.setGroup("ts");
					cfgItem.setKey(stock.getId());
				}
				
				// step 2.2 最大允许重试次数 
				int retry=0;
				while(retry<maxRetryNum) {
					try {
						Date start = new Date();
						crawlOneStock(stock.getId(), lastCrawlTs);
						cfgItem.setDate(start);
						configDao.addConfig(cfgItem);
						break;
					} catch (Exception e) {
						LOG.error("crawl exception, stock="+stock.getId()+", reason="+e.getClass().getName()+", msg="+e.getMessage());
						restartDriver();
						retry++;
					}
				}
				if(retry<maxRetryNum)
					LOG.info("succcess crawl stocktwis , stock="+stock.getId()+", day="+lastCrawlTs+", offset="+offset);
				else
					LOG.info("fail crawl stocktwis , stock="+stock.getId()+", day="+lastCrawlTs+", offset="+offset);
			}
			
			offset += limit;
			
			// 重启浏览器，清理缓存，防止内存泄露
			if(offset>0 && offset%1000==0) 
				restartDriver();
		}
		shutdownDriver();
		
		return 0;
	}
	
	
	protected void crawlOneStock(String stock, Date startDate) throws Exception {
		
		//super.restartDriver();
		
		String stockUrl = baseUrl+stock+"?q=\""+stock+"\"";
		driver.get(stockUrl);
		
		// step1, 解析信息量和和信息量波动
//		WebElement msgNumBtn = driver.findElement(By.cssSelector("#volume-tab"));
//		msgNumBtn.click();
//		WebElement msgNumElem = driver.findElement(By.cssSelector("#msg-volume .message-count"));
//		feedsInfo.setFeed_num(Integer.parseInt(msgNumElem.getText()));
		
		
		// step2, 抓取feeds
		Set<String> processedFeeds = new HashSet<String>();
		SimpleDateFormat sdf = new SimpleDateFormat("MMM. d  h:mm a yyyy", Locale.US);
		boolean hasNewFeeds = true;
				
		while(hasNewFeeds) {
		
			List<WebElement> feedElems = driver.findElements(By.cssSelector("#updates li.messageli div div.message"));
			hasNewFeeds=false;
			for(WebElement elem : feedElems) {
				String msgId = elem.getAttribute("id");
				if(processedFeeds.contains(msgId)) {
					continue;
				} else {
					hasNewFeeds = true;
				}
				
				StockFeedItem twisItem = new StockFeedItem();
				twisItem.setSrc(Constants.STOCKTWIS);
				
				String id = Constants.STOCKTWIS+"-"+elem.getAttribute("data-message-id");
				twisItem.setId(id);
				twisItem.setStock_id(stock);
				
				// step3.1  抓取正文
				WebElement msgBodyElem = elem.findElement(By.cssSelector(".message-body"));
				twisItem.setContent(msgBodyElem.getText());
				
				// step3.2  抓取作者名称
				WebElement nickElem = elem.findElement(By.cssSelector(".message-header a.username"));
				twisItem.setAuthor_name(nickElem.getText());
				// step3.3 抓取日期
				WebElement dateElem = elem.findElement(By.cssSelector(".message-date"));
				String timeStr = dateElem.getText().replace("at", "")+" "+Calendar.getInstance().get(Calendar.YEAR);
				try {
					twisItem.setTime(sdf.parse(timeStr));
				} catch (ParseException e) {
					LOG.error("parse time error, reason=", e);
					continue;
				} 
				
				if(twisItem.getTime().before(startDate)) {
					hasNewFeeds = false;
					break;
				}
				
		
				processedFeeds.add(msgId);
				// step3.4 判断情感
				if(twisItem.getContent().endsWith(BULLISH_TAG)) {
					twisItem.setAttitude(BULLISH_TAG);
				} else if(twisItem.getContent().endsWith(BEARISH_TAG)) {
					twisItem.setAttitude(BEARISH_TAG);
				} else {
					twisItem.setAttitude("");
				}
				
				stockFeedItemDao.addFeedItem(twisItem);
			}
			
			// 滚动屏幕
			scrollToBottom();
			// 判断是否加载结束，
			boolean isLoading = true;
			int retryNum = 10;
			while(isLoading && retryNum-->0) {
				try {
					Thread.sleep(1000);
				} catch (InterruptedException e) {
					;
				}
				
				if(driver.findElements(By.cssSelector("#more-button-loading")).size()>0
						&& driver.findElement(By.cssSelector("#more-button-loading")).isDisplayed())
					isLoading = true;
				else
					isLoading = false;
			}
		}
		
	}
	
	public String getBaseUrl() {
		return baseUrl;
	}

	public void setBaseUrl(String baseUrl) {
		this.baseUrl = baseUrl;
	}

}
